{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>short_description</th>\n",
       "      <th>headline</th>\n",
       "      <th>date</th>\n",
       "      <th>link</th>\n",
       "      <th>authors</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>She left her husband. He killed their children...</td>\n",
       "      <td>There Were 2 Mass Shootings In Texas Last Week...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/texas-ama...</td>\n",
       "      <td>Melissa Jeltsen</td>\n",
       "      <td>CRIME</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Of course it has a song.</td>\n",
       "      <td>Will Smith Joins Diplo And Nicky Jam For The 2...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/will-smit...</td>\n",
       "      <td>Andy McDonald</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The actor and his longtime girlfriend Anna Ebe...</td>\n",
       "      <td>Hugh Grant Marries For The First Time At Age 57</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/hugh-gran...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The actor gives Dems an ass-kicking for not fi...</td>\n",
       "      <td>Jim Carrey Blasts 'Castrato' Adam Schiff And D...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/jim-carre...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The \"Dietland\" actress said using the bags is ...</td>\n",
       "      <td>Julianna Margulies Uses Donald Trump Poop Bags...</td>\n",
       "      <td>2018-05-26</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/julianna-...</td>\n",
       "      <td>Ron Dicker</td>\n",
       "      <td>ENTERTAINMENT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   short_description  \\\n",
       "0  She left her husband. He killed their children...   \n",
       "1                           Of course it has a song.   \n",
       "2  The actor and his longtime girlfriend Anna Ebe...   \n",
       "3  The actor gives Dems an ass-kicking for not fi...   \n",
       "4  The \"Dietland\" actress said using the bags is ...   \n",
       "\n",
       "                                            headline       date  \\\n",
       "0  There Were 2 Mass Shootings In Texas Last Week... 2018-05-26   \n",
       "1  Will Smith Joins Diplo And Nicky Jam For The 2... 2018-05-26   \n",
       "2    Hugh Grant Marries For The First Time At Age 57 2018-05-26   \n",
       "3  Jim Carrey Blasts 'Castrato' Adam Schiff And D... 2018-05-26   \n",
       "4  Julianna Margulies Uses Donald Trump Poop Bags... 2018-05-26   \n",
       "\n",
       "                                                link          authors  \\\n",
       "0  https://www.huffingtonpost.com/entry/texas-ama...  Melissa Jeltsen   \n",
       "1  https://www.huffingtonpost.com/entry/will-smit...    Andy McDonald   \n",
       "2  https://www.huffingtonpost.com/entry/hugh-gran...       Ron Dicker   \n",
       "3  https://www.huffingtonpost.com/entry/jim-carre...       Ron Dicker   \n",
       "4  https://www.huffingtonpost.com/entry/julianna-...       Ron Dicker   \n",
       "\n",
       "        category  \n",
       "0          CRIME  \n",
       "1  ENTERTAINMENT  \n",
       "2  ENTERTAINMENT  \n",
       "3  ENTERTAINMENT  \n",
       "4  ENTERTAINMENT  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_json('News_Category_Dataset.json', lines = True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将数据按照字段'category‘分类\n",
    "categories = df.groupby('category')\n",
    "df.category = df.category.map(lambda x:\"WORLDPOST\" if x == \"THE WORLDPOST\" else x)\n",
    "print(\"total categories: \", categories.ngroups)\n",
    "print(categories.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "#显示所有列\n",
    "pd.set_option('display.max_columns', None)\n",
    "#显示所有行\n",
    "pd.set_option('display.max_rows', None)\n",
    "#设置value的显示长度为100，默认为50\n",
    "pd.set_option('max_colwidth',None)\n",
    "\n",
    "df['text'] = df.headline + \" \" + df.short_description\n",
    "print(df.text[0:10])\n",
    "#将单词进行标号\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(df.text)\n",
    "X = tokenizer.texts_to_sequences(df.text)\n",
    "df['words'] = X\n",
    "print(X[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#记录每条数据的单词数\n",
    "df['word_length'] = df.words.apply(lambda i:len(i))\n",
    "#q清除igchu"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python [conda env:keras1] *",
   "language": "python",
   "name": "conda-env-keras1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
